#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Date    : 2018-08-15 14:57:21
# @Author  : yangchaojun (YYChildren@gmail.com)
# @Link    : https://git.mingchao.com/yangchaojun
# @Version :

from mtmodel.utils import pd_source, misc
from mtmodel.utils.config import CONFIG
from mtmodel.model import text_process
from tqdm import tqdm
tqdm.pandas()
import bottleneck as bn


from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import LatentDirichletAllocation

import logging
logging.basicConfig(level=logging.INFO)

def print_top_words(model, feature_names, n_top_words):
    nk = bn.argpartition(-model.components_, n_top_words)[:, :n_top_words]
    for i in range(len(model.components_)):
        top = [ (k,model.components_[i][k]) for k in nk[i]]
        top.sort(key=lambda x:x[1], reverse=True)
        print("Topic #%d:" % i)
        print(" ".join([feature_names[t] for t,_ in top]))

def train_lda(words, n_components=256, max_iter=50, verbose = 1):
    tfidf_vectorizer=TfidfVectorizer(analyzer = lambda x:x)
    tfidf_vectorizer.fit(words)
    tfidf = tfidf_vectorizer.transform(words)
    lda = LatentDirichletAllocation(n_components=n_components, max_iter=max_iter, verbose = verbose)
    lda.fit(tfidf)
    return tfidf_vectorizer,lda

def create_game_lda():
    sql = "select d.source,d.game_id,d.game_name,d.description,d.category from game_source.s_game_detail d WHERE d.source ='taptap'"
    df_detail = pd_source.from_sql(sql)
    ## 
    df_detail['description'] = df_detail['description'].progress_apply(
        text_process.clear_text)
    ## 删除非中文游戏
    df_detail = df_detail[df_detail['description'].progress_apply(
        text_process.is_chinese_sentence)].reset_index()
    ## 提取关键词
    df_detail['words'] = df_detail['description'].progress_apply(
        text_process.extract_textrank_keywords, topK=None)
    tfidf_vectorizer,lda = train_lda(df_detail['words'])
    tfidf_model_path = misc.get_sklearn_path(CONFIG['game_lda']['tfidf_model'])
    lda_model_path = misc.get_sklearn_path(CONFIG['game_lda']['lda_model'])
    misc.dump(tfidf_vectorizer, tfidf_model_path)
    misc.dump(lda, lda_model_path)

def main():
    create_game_lda()

if __name__ == '__main__':
    main()
